
import codecs
import sys
import glob
import re
import pepe.p_txt.bigtxt as bt

for f in glob.glob('*.dat'):

    i = codecs.open(f, 'rb', 'utf-8')
    o = codecs.open(f + '_modify.#', 'wb', 'utf-8')

    lines = i.readlines()

    for line in lines:
        line = re.sub(' {0,}\" {0,}\x0d\x0A', '|\x0d\x0A', line)
        line = re.sub('^\"', '', line)
        line = re.sub('\xEF\xBB\xBF\"', '\xEF\xBB\xBF', line)

        line = re.sub(' {0,}\"\|\" {0,}', '|', line)
        line = re.sub(' {2}', '', line)

        o.write(line)

    o.close()

